library(NNbenchmark)
library(ANN2)

library(stringr)
library(dplyr)
library(kableExtra)

Set Environment

options(scipen = 9999)
options("digits.secs" = 2)
timer  <- createTimer(verbose = FALSE)

Datasets to Test

NNdataSummary(NNdatasets)

NN Train Function

hyperParams <- function(optim_method) {
    
    if (!is.element(optim_method, c("sgd", "adam", "rmsprop"))) stop("Invalid Parameters.")
    if (optim_method == "sgd") { iter <- 10; lr <- 0.01; params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)} 
    if (optim_method == "adam") { iter <- 15; lr <- 0.02; params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)} 
    if (optim_method == "rmsprop") { iter <- 20; lr <- 0.03; params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)} 
    
    params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)
    
    out <- list(iter = iter, lr = lr, params = params)
    
    return (out)
}



NNtrain <- function(x, y, hidden_neur, optim_method) {
    
    hyper_params <- hyperParams(optim_method)
    
    iter <- hyper_params$iter
    lr <- hyper_params$lr
    
    NNreg <- neuralnetwork(X = x, y = y, 
                           val.prop = 0, 
                           standardize = FALSE, 
                           hidden.layers = c(hidden_neur), 
                           regression = TRUE,
                           loss.type = "squared",
                           n.epochs = iter,
                           optim.type = optim_method,
                           learn.rates = lr,
                           verbose = FALSE,
                           random.seed = as.integer(runif(1)*10000000))
    
    return (NNreg)
}

Main Loop

for (dset in names(NNdatasets)) {

    ## =============================================
    ## EXTRACT INFORMATION FROM THE SELECTED DATASET
    ## =============================================
    ds     <- NNdatasets[[dset]]$ds
    Z      <- NNdatasets[[dset]]$Z
    neur   <- NNdatasets[[dset]]$neur
    nparNN <- NNdatasets[[dset]]$nparNN
    fmlaNN <- NNdatasets[[dset]]$fmlaNN
    donotremove  <- c("dset", "dsets", "ds", "Z", "neur", "TF", "nrep", "timer",
                      "donotremove", "donotremove2")
    donotremove2 <- c("dset", "dsets") 

    ## ===================================================
    ## SELECT THE FORMAT REQUIRED BY THE PACKAGE/ALGORITHM
    ## d = data.frame, m = matrix, v = vector/numeric
    ## ATTACH THE OBJECTS CREATED (x, y, Zxy, ... )
    ## ===================================================
    ZZ     <- prepareZZ(Z, xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
    attach(ZZ)

    ## =============================================
    ## SELECT THE PACKAGE USED FOR TRAINING
    ## nrep => SELECT THE NUMBER OF INDEPENDANT RUNS
    ## iter => SELECT THE MAX NUMBER OF ITERATIONS
    ## TF   => PLOT THE RESULTS
    ## =============================================

    
    nrep   <- 5
    TF     <- TRUE 

    method <- c("sgd", "adam", "rmsprop")
        
    for (m in method) {
        
        descr  <- paste(dset, "ANN2::neuralnetwork", m, sep = "_")

        ## AUTO
        Ypred  <- list()
        Rmse   <- numeric(length = nrep)
        Mae    <- numeric(length = nrep)
    
        for(i in 1:nrep){
            event      <- paste0(descr, sprintf("_%.2d", i))
            timer$start(event)
            #### ADJUST THE FOLLOWING LINES TO THE PACKAGE::ALGORITHM
            
            hyper_params <- hyperParams(optim_method = m)

            NNreg      <- tryCatch(
                            NNtrain(x = x, y = y, hidden_neur = neur, optim_method = m),
                            error = function(y) {lm(y ~ 0, data = Zxy)}
                          )     
            y_pred     <- tryCatch(
                            ym0 + ysd0*predict(NNreg, x)$predictions,
                            error = ym0
                          )     
            ####
            Ypred[[i]] <- y_pred
            Rmse[i]    <- funRMSE(y_pred, y0)
            Mae[i]     <- funMAE(y_pred, y0)
            timer$stop(event, RMSE = Rmse[i], MAE = Mae[i], params = hyper_params$params, printmsg = FALSE)
        }
        best <- which(Rmse == min(Rmse, na.rm = TRUE))[1]
        best ; Rmse[[best]]
        
        ## ================================================
        ## PLOT ALL MODELS AND THE MODEL WITH THE BEST RMSE
        ## par OPTIONS CAN BE IMPROVED FOR A BETTER DISPLAY
        ## ================================================
        op <- par(mfcol = c(1,2))
        plotNN(xory, y0, uni, TF, main = descr)
        for (i in 1:nrep) lipoNN(xory, Ypred[[i]], uni, TF, col = i, lwd = 1)
        
        plotNN(xory, y0, uni, TF, main = descr)
        lipoNN(xory, Ypred[[best]], uni, TF, col = 4, lwd = 4)
        par(op)
    }


## ===========================
## DETACH ZZ - END OF THE LOOP
## ===========================
    detach(ZZ)
}

Results

dfr0 <- getTimer(timer) 

dfr  <- data.frame(
    ds_pkg.fun_algo = stringr::str_sub(dfr0[ ,1], 1, -4),
    run     = stringr::str_sub(dfr0[ ,1], -2, -1),
    dataset = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "^\\w*_"), fixed("_"), ""),
    method = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "_\\w*_"), fixed("_"), ""),
    Elapsed = round(dfr0[ ,4], 5),
    params = dfr0$params,
    dfr0[, c("RMSE","MAE")]
)


dfr

Best Results

dfr %>%
    group_by(dataset, method) %>%
    summarise(minRMSE = min(RMSE), meanRMSE = mean(RMSE), meanTime = mean(Elapsed)) %>%
    kable() %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive")) %>%
    collapse_rows(columns = 1:2, valign = "top")
dataset method minRMSE meanRMSE meanTime
mDette adam 3.0948 3.97626 0.008
rmsprop 2.8776 3.36848 0.010
sgd 4.9184 6.50956 0.028
mFriedman adam 0.0808 0.11418 0.012
rmsprop 0.0591 0.09248 0.010
sgd 0.1261 0.13998 0.010
mIshigami adam 2.7179 2.76180 0.014
rmsprop 2.3168 2.67236 0.020
sgd 2.9226 3.07040 0.008
mRef153 adam 6.0301 7.10828 0.010
rmsprop 4.6908 5.95992 0.006
sgd 7.5511 8.13686 0.006
uDmod1 adam 0.5276 0.57870 0.006
rmsprop 0.4809 0.51358 0.008
sgd 0.5673 0.69250 0.006
uDmod2 adam 0.3298 0.48506 0.004
rmsprop 0.3885 0.44276 0.006
sgd 0.4324 0.48176 0.010
uDreyfus1 adam 0.6140 0.68476 0.004
rmsprop 0.4876 0.59500 0.000
sgd 0.6968 0.80534 0.002
uDreyfus2 adam 0.6809 0.90494 0.004
rmsprop 0.4592 0.52160 0.000
sgd 0.6762 0.73370 0.004
uGauss1 adam 26.7491 27.76072 0.010
rmsprop 22.7846 25.61874 0.004
sgd 27.5268 28.25596 0.004
uGauss2 adam 19.2962 21.46538 0.008
rmsprop 13.2070 14.95164 0.016
sgd 17.3750 23.07154 0.002
uGauss3 adam 17.2794 20.72808 0.004
rmsprop 12.5749 14.10714 0.004
sgd 23.3915 27.76456 0.006
uNeuroOne adam 0.9157 1.02970 0.002
rmsprop 0.8519 0.95704 0.004
sgd 0.9877 1.09072 0.002
clearNN(donotremove)
LS0tCnRpdGxlOiAiTk5iZW5jaG1hcmsgfCBBTk4yIgphdXRob3I6IEFrc2hhaiBWZXJtYQpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgoKYGBge3IgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0KbGlicmFyeShOTmJlbmNobWFyaykKbGlicmFyeShBTk4yKQoKbGlicmFyeShzdHJpbmdyKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KGthYmxlRXh0cmEpCmBgYAoKIyMgU2V0IEVudmlyb25tZW50CmBgYHtyfQpvcHRpb25zKHNjaXBlbiA9IDk5OTkpCm9wdGlvbnMoImRpZ2l0cy5zZWNzIiA9IDIpCnRpbWVyICA8LSBjcmVhdGVUaW1lcih2ZXJib3NlID0gRkFMU0UpCmBgYAoKIyMgRGF0YXNldHMgdG8gVGVzdAoKYGBge3J9Ck5OZGF0YVN1bW1hcnkoTk5kYXRhc2V0cykKYGBgCgojIyBOTiBUcmFpbiBGdW5jdGlvbgpgYGB7cn0KaHlwZXJQYXJhbXMgPC0gZnVuY3Rpb24ob3B0aW1fbWV0aG9kKSB7CiAgICAKICAgIGlmICghaXMuZWxlbWVudChvcHRpbV9tZXRob2QsIGMoInNnZCIsICJhZGFtIiwgInJtc3Byb3AiKSkpIHN0b3AoIkludmFsaWQgUGFyYW1ldGVycy4iKQogICAgaWYgKG9wdGltX21ldGhvZCA9PSAic2dkIikgeyBpdGVyIDwtIDEwOyBsciA8LSAwLjAxOyBwYXJhbXMgPC0gcGFzdGUwKCJtZXRob2Q9Iiwgb3B0aW1fbWV0aG9kLCAiXyIsICJscj0iLCBsciwgIl8iLCAiaXRlcj0iLCBpdGVyKX0gCiAgICBpZiAob3B0aW1fbWV0aG9kID09ICJhZGFtIikgeyBpdGVyIDwtIDE1OyBsciA8LSAwLjAyOyBwYXJhbXMgPC0gcGFzdGUwKCJtZXRob2Q9Iiwgb3B0aW1fbWV0aG9kLCAiXyIsICJscj0iLCBsciwgIl8iLCAiaXRlcj0iLCBpdGVyKX0gCiAgICBpZiAob3B0aW1fbWV0aG9kID09ICJybXNwcm9wIikgeyBpdGVyIDwtIDIwOyBsciA8LSAwLjAzOyBwYXJhbXMgPC0gcGFzdGUwKCJtZXRob2Q9Iiwgb3B0aW1fbWV0aG9kLCAiXyIsICJscj0iLCBsciwgIl8iLCAiaXRlcj0iLCBpdGVyKX0gCiAgICAKICAgIHBhcmFtcyA8LSBwYXN0ZTAoIm1ldGhvZD0iLCBvcHRpbV9tZXRob2QsICJfIiwgImxyPSIsIGxyLCAiXyIsICJpdGVyPSIsIGl0ZXIpCiAgICAKICAgIG91dCA8LSBsaXN0KGl0ZXIgPSBpdGVyLCBsciA9IGxyLCBwYXJhbXMgPSBwYXJhbXMpCiAgICAKICAgIHJldHVybiAob3V0KQp9CgoKCk5OdHJhaW4gPC0gZnVuY3Rpb24oeCwgeSwgaGlkZGVuX25ldXIsIG9wdGltX21ldGhvZCkgewogICAgCiAgICBoeXBlcl9wYXJhbXMgPC0gaHlwZXJQYXJhbXMob3B0aW1fbWV0aG9kKQogICAgCiAgICBpdGVyIDwtIGh5cGVyX3BhcmFtcyRpdGVyCiAgICBsciA8LSBoeXBlcl9wYXJhbXMkbHIKICAgIAogICAgTk5yZWcgPC0gbmV1cmFsbmV0d29yayhYID0geCwgeSA9IHksIAogICAgICAgICAgICAgICAgICAgICAgICAgICB2YWwucHJvcCA9IDAsIAogICAgICAgICAgICAgICAgICAgICAgICAgICBzdGFuZGFyZGl6ZSA9IEZBTFNFLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgaGlkZGVuLmxheWVycyA9IGMoaGlkZGVuX25ldXIpLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgcmVncmVzc2lvbiA9IFRSVUUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIGxvc3MudHlwZSA9ICJzcXVhcmVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgbi5lcG9jaHMgPSBpdGVyLAogICAgICAgICAgICAgICAgICAgICAgICAgICBvcHRpbS50eXBlID0gb3B0aW1fbWV0aG9kLAogICAgICAgICAgICAgICAgICAgICAgICAgICBsZWFybi5yYXRlcyA9IGxyLAogICAgICAgICAgICAgICAgICAgICAgICAgICB2ZXJib3NlID0gRkFMU0UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIHJhbmRvbS5zZWVkID0gYXMuaW50ZWdlcihydW5pZigxKSoxMDAwMDAwMCkpCiAgICAKICAgIHJldHVybiAoTk5yZWcpCn0KYGBgCgoKIyMgTWFpbiBMb29wCmBgYHtyIGZpZy5oZWlnaHQ9NSwgZmlnLndpZHRoPTE0fQpmb3IgKGRzZXQgaW4gbmFtZXMoTk5kYXRhc2V0cykpIHsKCiAgICAjI8KgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiAgICAjI8KgRVhUUkFDVCBJTkZPUk1BVElPTiBGUk9NIFRIRSBTRUxFQ1RFRCBEQVRBU0VUCiAgICAjI8KgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiAgICBkcyAgICAgPC0gTk5kYXRhc2V0c1tbZHNldF1dJGRzCiAgICBaICAgICAgPC0gTk5kYXRhc2V0c1tbZHNldF1dJFoKICAgIG5ldXIgICA8LSBOTmRhdGFzZXRzW1tkc2V0XV0kbmV1cgogICAgbnBhck5OIDwtIE5OZGF0YXNldHNbW2RzZXRdXSRucGFyTk4KICAgIGZtbGFOTiA8LSBOTmRhdGFzZXRzW1tkc2V0XV0kZm1sYU5OCiAgICBkb25vdHJlbW92ZSAgPC0gYygiZHNldCIsICJkc2V0cyIsICJkcyIsICJaIiwgIm5ldXIiLCAiVEYiLCAibnJlcCIsICJ0aW1lciIsCiAgICAgICAgICAgICAgICAgICAgICAiZG9ub3RyZW1vdmUiLCAiZG9ub3RyZW1vdmUyIikKICAgIGRvbm90cmVtb3ZlMiA8LSBjKCJkc2V0IiwgImRzZXRzIikgCgoKCiAgICAjI8KgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiAgICAjIyBTRUxFQ1QgVEhFIEZPUk1BVCBSRVFVSVJFRCBCWSBUSEUgUEFDS0FHRS9BTEdPUklUSE0KICAgICMjIGQgPSBkYXRhLmZyYW1lLCBtID0gbWF0cml4LCB2ID0gdmVjdG9yL251bWVyaWMKICAgICMjwqBBVFRBQ0ggVEhFIE9CSkVDVFMgQ1JFQVRFRCAoeCwgeSwgWnh5LCAuLi4gKQogICAgIyMgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiAgICBaWiAgICAgPC0gcHJlcGFyZVpaKFosIHhkbXYgPSAiZCIsIHlkbXYgPSAidiIsIHpkbSA9ICJkIiwgc2NhbGUgPSBUUlVFKQogICAgYXR0YWNoKFpaKQoKICAgICMjwqA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KICAgICMjwqBTRUxFQ1QgVEhFIFBBQ0tBR0UgVVNFRCBGT1IgVFJBSU5JTkcKICAgICMjIG5yZXAgPT4gU0VMRUNUIFRIRSBOVU1CRVIgT0YgSU5ERVBFTkRBTlQgUlVOUwogICAgIyPCoGl0ZXIgPT4gU0VMRUNUIFRIRSBNQVggTlVNQkVSIE9GIElURVJBVElPTlMKICAgICMjwqBURiAgID0+IFBMT1QgVEhFIFJFU1VMVFMKICAgICMjwqA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiAgICAKICAgIG5yZXAgICA8LSA1CiAgICBURiAgICAgPC0gVFJVRSAKCiAgICBtZXRob2QgPC0gYygic2dkIiwgImFkYW0iLCAicm1zcHJvcCIpCiAgICAgICAgCiAgICBmb3IgKG0gaW4gbWV0aG9kKSB7CiAgICAgICAgCiAgICAgICAgZGVzY3IgIDwtIHBhc3RlKGRzZXQsICJBTk4yOjpuZXVyYWxuZXR3b3JrIiwgbSwgc2VwID0gIl8iKQoKICAgICAgICAjI8KgQVVUTwogICAgICAgIFlwcmVkICA8LSBsaXN0KCkKICAgICAgICBSbXNlICAgPC0gbnVtZXJpYyhsZW5ndGggPSBucmVwKQogICAgICAgIE1hZSAgICA8LSBudW1lcmljKGxlbmd0aCA9IG5yZXApCiAgICAKICAgICAgICBmb3IoaSBpbiAxOm5yZXApewogICAgICAgICAgICBldmVudCAgICAgIDwtIHBhc3RlMChkZXNjciwgc3ByaW50ZigiXyUuMmQiLCBpKSkKICAgICAgICAgICAgdGltZXIkc3RhcnQoZXZlbnQpCiAgICAgICAgICAgICMjIyMgQURKVVNUIFRIRSBGT0xMT1dJTkcgTElORVMgVE8gVEhFIFBBQ0tBR0U6OkFMR09SSVRITQogICAgICAgICAgICAKICAgICAgICAgICAgaHlwZXJfcGFyYW1zIDwtIGh5cGVyUGFyYW1zKG9wdGltX21ldGhvZCA9IG0pCgogICAgICAgICAgICBOTnJlZyAgICAgIDwtIHRyeUNhdGNoKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgTk50cmFpbih4ID0geCwgeSA9IHksIGhpZGRlbl9uZXVyID0gbmV1ciwgb3B0aW1fbWV0aG9kID0gbSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBlcnJvciA9IGZ1bmN0aW9uKHkpIHtsbSh5IH4gMCwgZGF0YSA9IFp4eSl9CiAgICAgICAgICAgICAgICAgICAgICAgICAgKSAgICAgCiAgICAgICAgICAgIHlfcHJlZCAgICAgPC0gdHJ5Q2F0Y2goCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB5bTAgKyB5c2QwKnByZWRpY3QoTk5yZWcsIHgpJHByZWRpY3Rpb25zLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgZXJyb3IgPSB5bTAKICAgICAgICAgICAgICAgICAgICAgICAgICApICAgICAKICAgICAgICAgICAgIyMjIwogICAgICAgICAgICBZcHJlZFtbaV1dIDwtIHlfcHJlZAogICAgICAgICAgICBSbXNlW2ldICAgIDwtIGZ1blJNU0UoeV9wcmVkLCB5MCkKICAgICAgICAgICAgTWFlW2ldICAgICA8LSBmdW5NQUUoeV9wcmVkLCB5MCkKICAgICAgICAgICAgdGltZXIkc3RvcChldmVudCwgUk1TRSA9IFJtc2VbaV0sIE1BRSA9IE1hZVtpXSwgcGFyYW1zID0gaHlwZXJfcGFyYW1zJHBhcmFtcywgcHJpbnRtc2cgPSBGQUxTRSkKICAgICAgICB9CiAgICAgICAgYmVzdCA8LSB3aGljaChSbXNlID09IG1pbihSbXNlLCBuYS5ybSA9IFRSVUUpKVsxXQogICAgICAgIGJlc3QgOyBSbXNlW1tiZXN0XV0KICAgICAgICAKICAgICAgICAjIyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KICAgICAgICAjI8KgUExPVCBBTEwgTU9ERUxTIEFORCBUSEUgTU9ERUwgV0lUSCBUSEUgQkVTVCBSTVNFCiAgICAgICAgIyPCoHBhciBPUFRJT05TIENBTiBCRSBJTVBST1ZFRCBGT1IgQSBCRVRURVIgRElTUExBWQogICAgICAgICMjID09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQogICAgICAgIG9wIDwtIHBhcihtZmNvbCA9IGMoMSwyKSkKICAgICAgICBwbG90Tk4oeG9yeSwgeTAsIHVuaSwgVEYsIG1haW4gPSBkZXNjcikKICAgICAgICBmb3IgKGkgaW4gMTpucmVwKSBsaXBvTk4oeG9yeSwgWXByZWRbW2ldXSwgdW5pLCBURiwgY29sID0gaSwgbHdkID0gMSkKICAgICAgICAKICAgICAgICBwbG90Tk4oeG9yeSwgeTAsIHVuaSwgVEYsIG1haW4gPSBkZXNjcikKICAgICAgICBsaXBvTk4oeG9yeSwgWXByZWRbW2Jlc3RdXSwgdW5pLCBURiwgY29sID0gNCwgbHdkID0gNCkKICAgICAgICBwYXIob3ApCiAgICB9CgoKIyPCoD09PT09PT09PT09PT09PT09PT09PT09PT09PQojIyBERVRBQ0ggWlogLSBFTkQgT0YgVEhFIExPT1AKIyPCoD09PT09PT09PT09PT09PT09PT09PT09PT09PQogICAgZGV0YWNoKFpaKQp9CmBgYAoKCiMjIFJlc3VsdHMKCmBgYHtyfQpkZnIwIDwtIGdldFRpbWVyKHRpbWVyKSAKCmRmciAgPC0gZGF0YS5mcmFtZSgKICAgIGRzX3BrZy5mdW5fYWxnbyA9IHN0cmluZ3I6OnN0cl9zdWIoZGZyMFsgLDFdLCAxLCAtNCksCiAgICBydW4gICAgID0gc3RyaW5ncjo6c3RyX3N1YihkZnIwWyAsMV0sIC0yLCAtMSksCiAgICBkYXRhc2V0ID0gc3RyaW5ncjo6c3RyX3JlcGxhY2VfYWxsKHN0cmluZ3I6OnN0cl9leHRyYWN0KGRmcjBbLCAxXSwgcGF0dGVybiA9ICJeXFx3Kl8iKSwgZml4ZWQoIl8iKSwgIiIpLAogICAgbWV0aG9kID0gc3RyaW5ncjo6c3RyX3JlcGxhY2VfYWxsKHN0cmluZ3I6OnN0cl9leHRyYWN0KGRmcjBbLCAxXSwgcGF0dGVybiA9ICJfXFx3Kl8iKSwgZml4ZWQoIl8iKSwgIiIpLAogICAgRWxhcHNlZCA9IHJvdW5kKGRmcjBbICw0XSwgNSksCiAgICBwYXJhbXMgPSBkZnIwJHBhcmFtcywKICAgIGRmcjBbLCBjKCJSTVNFIiwiTUFFIildCikKCgpkZnIKYGBgCgojIyBCZXN0IFJlc3VsdHMKCmBgYHtyfQpkZnIgJT4lCiAgICBncm91cF9ieShkYXRhc2V0LCBtZXRob2QpICU+JQogICAgc3VtbWFyaXNlKG1pblJNU0UgPSBtaW4oUk1TRSksIG1lYW5STVNFID0gbWVhbihSTVNFKSwgbWVhblRpbWUgPSBtZWFuKEVsYXBzZWQpKSAlPiUKICAgIGthYmxlKCkgJT4lCiAgICBrYWJsZV9zdHlsaW5nKGJvb3RzdHJhcF9vcHRpb25zID0gYygic3RyaXBlZCIsICJob3ZlciIsICJjb25kZW5zZWQiLCAicmVzcG9uc2l2ZSIpKSAlPiUKICAgIGNvbGxhcHNlX3Jvd3MoY29sdW1ucyA9IDE6MiwgdmFsaWduID0gInRvcCIpCmBgYAoKYGBge3J9CmNsZWFyTk4oZG9ub3RyZW1vdmUpCmBgYAoK